#!/usr/bin/env python
# encoding: utf-8
import random
import redis
import sys
import os
from lxml import etree
sys.path.append(os.getcwd())

from DangDangCrawl.settings import LOCAL_REDIS_HOST, LOCAL_REDIS_PORT
import requests

r = redis.Redis(host=LOCAL_REDIS_HOST, port=LOCAL_REDIS_PORT)

u='http://category.dangdang.com/?ref=www-0-C'
html=requests.get(url=u,headers={'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.85 Safari/537.36'
                                     ,'Host': 'category.dangdang.com',
                                     'Cookie': 'from=460-5-biaoti; __permanent_id=20210420172552168106066184286480932; __ddc_15d_f=1618910752%7C!%7C_utm_brand_id%3D11106; ddscreen=2; __rpm=%7Cs_9494.2672734.1.1619068693261; search_passback=a86800891a712cd2110781600000000005d5c80002078160; pos_9_end=1619068713243; pos_6_end=1619068713382; pos_6_start=1619069013365; order_follow_source=P-460-5-bi%7C%231%7C%23www.baidu.com%252Fother.php%253Fsc.060000j_e2P9R1IHGxPmzwGNLgKBKpx1hRe5Yuhd2mjKhltblmB9ot59k2iaf-LB-IeCOEHsh%7C%230-%7C-; dest_area=country_id%3D9000%26province_id%3D111%26city_id%20%3D0%26district_id%3D0%26town_id%3D0; __visit_id=20210423100035985133676433652715981; __out_refer=1619143236%7C!%7Cwww.baidu.com%7C!%7C%25E5%25BD%2593%25E5%25BD%2593%25E7%25BD%2591; __ddc_1d=1619143236%7C!%7C_utm_brand_id%3D11106; __ddc_24h=1619143236%7C!%7C_utm_brand_id%3D11106; __ddc_15d=1619143236%7C!%7C_utm_brand_id%3D11106; __trace_id=20210423100043251354397485378471148'

}
                    ).text
response=etree.HTML(html)
div_list = response.xpath('//div[@class="classify_books"]/div[@class="classify_kind"]')
for div in div_list[10:12]:
    item = {}
    item["大标题"] = div.xpath('.//a/项目必看.txt()')
    li_list = div.xpath('.//ul[@class="classify_kind_detail"]/li')
    for li in li_list:
        aa= li.xpath('./a/项目必看.txt()')[0]
        sm_url = li.xpath('./a/@href')[0]
        if sm_url != "javascript:void(0);":
            # print(sm_url, item["小标题"])
            print(aa)
            print(sm_url)

            r.lpush('DangDang:d_url', sm_url)

        break
